#! /usr/bin/perl/ -w
use strict;
use warnings;
use LWP::Simple;

#given an url, return page content
sub getPageContent{
	my $url = shift;
	my $page=get($url);
	return $page;
}

#given page content, return all <a> href links includes(follow(default), nofollow)
sub getAllLinks{
	my $content = shift;
	my @links = $content =~ /<a.*?href\s*=\s*"(http[^"\s\>]+)/gis;
	my $alllinks = join("\n",@links);
	return $alllinks
}

# given a page content, return all links in <a> tag without nofollow
sub getAllFollowLinks{
	my $content = shift;
	my @nofollowLinks = $content =~ /<a.*?rel\s*=\s*"nofollow".*?href\s*=\s*"(http[^"\s\>]+).*?>/gis;
    my @nofollowLinks2 = $content =~ /<a.*?href\s*=\s*"(http[^"\s\>]+).*?rel\s*=\s*"nofollow".*?>/gis;
    my @allLinks = $content =~ /<a.*?href\s*=\s*"(http[^"\s\>]+)/gis;
    my @result=();
    my %nofollowHash=();
	$nofollowHash{$_}++ for(@nofollowLinks, @nofollowLinks2);
	for(my $i=0; $i<scalar(@allLinks); $i++){
		if(exists($nofollowHash{$allLinks[$i]})){
			next;
		}
		$nofollowHash{$allLinks[$i]}++;
		push(@result, $allLinks[$i]);
	}
	my $resultStr = join("\n", @result);
	return $resultStr;
}

#test code bellow
#my $pageContent = getPageContent("http://www.alibaba.com/countrysearch/CN/mp3.html");
#print getAllFollowLinks($pageContent)."\n";

1;
